{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ed161202",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/usr/local/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/usr/local/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/usr/local/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:528: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/usr/local/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:529: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/usr/local/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:530: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/usr/local/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:535: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using GPU for acceleration\n",
      "Input data: mnist\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/usr/local/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-0.1922, -0.1843, -0.1608,  ..., -0.2392, -0.2392, -0.2314],\n",
      "        [-0.2627, -0.2627, -0.2471,  ..., -0.2157, -0.2235, -0.2235],\n",
      "        [-0.3020, -0.3098, -0.3098,  ..., -0.1686, -0.1922, -0.2000],\n",
      "        ...,\n",
      "        [-0.9294, -0.9294, -0.9451,  ..., -0.8039, -0.8039, -0.7961],\n",
      "        [-0.8431, -0.8667, -0.8902,  ..., -0.8039, -0.7961, -0.7882],\n",
      "        [-0.8039, -0.8039, -0.8039,  ..., -0.8275, -0.8353, -0.8353]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.9922, -0.9922, -0.9922,  ..., -0.9529, -0.1765, -0.9922],\n",
      "        [-0.9922, -0.9922, -0.9922,  ..., -0.9529, -0.1765, -0.9843],\n",
      "        [-0.9922, -0.9922, -0.9922,  ..., -0.9608, -0.1765, -0.9922],\n",
      "        ...,\n",
      "        [-0.5922, -0.6000, -0.5608,  ..., -0.7882, -0.8196, -0.8039],\n",
      "        [-0.6000, -0.5608, -0.5529,  ..., -0.8039, -0.7647, -0.7725],\n",
      "        [-0.5294, -0.5922, -0.5922,  ..., -0.7647, -0.7961, -0.8431]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.7961, -0.8745,  0.4824,  ...,  0.6314,  0.7098,  0.7569],\n",
      "        [-0.9373, -0.9686,  0.1686,  ...,  0.6000,  0.6078,  0.6157],\n",
      "        [-0.9922, -0.9843,  0.2627,  ...,  0.3569,  0.3961,  0.3647],\n",
      "        ...,\n",
      "        [-0.7725, -0.7961, -0.8118,  ..., -0.8039, -0.8196, -0.8039],\n",
      "        [-0.7961, -0.7647, -0.7647,  ..., -0.8196, -0.8039, -0.7490],\n",
      "        [-0.8039, -0.7647, -0.7490,  ..., -0.8275, -0.8118, -0.7725]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.9922, -0.9922, -0.9922,  ...,  0.5059,  0.4118, -1.0000],\n",
      "        [-0.9922, -0.9922, -0.9922,  ...,  0.3412,  0.2941,  0.2235],\n",
      "        [-0.9922, -0.9922, -0.9922,  ...,  0.3176,  0.3098,  0.2863],\n",
      "        ...,\n",
      "        [-0.7569, -0.7176, -0.7020,  ..., -0.8353, -0.8588, -0.8745],\n",
      "        [-0.7490, -0.7804, -0.7647,  ..., -0.8431, -0.8353, -0.8275],\n",
      "        [-0.7569, -0.7647, -0.7490,  ..., -0.8353, -0.8431, -0.8902]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.7020,  0.4353,  0.5765,  ...,  0.3098,  0.3412,  0.2549],\n",
      "        [ 0.4039, -0.6863, -0.9529,  ...,  0.0353,  0.0667,  0.1137],\n",
      "        [ 0.5765, -0.9608, -0.9294,  ...,  0.2078,  0.2000,  0.2000],\n",
      "        ...,\n",
      "        [-0.3725, -0.3255, -0.2627,  ...,  0.0745,  0.0431,  0.0980],\n",
      "        [-0.3098, -0.2941, -0.2392,  ..., -0.0353, -0.0118,  0.0431],\n",
      "        [-0.2627, -0.3020, -0.2784,  ..., -0.0275,  0.0510,  0.0588]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.5216, -0.8431, -0.5451,  ...,  0.7882,  0.7647,  0.8118],\n",
      "        [-0.4118, -0.8118,  0.4196,  ...,  0.7020,  0.7020,  0.7255],\n",
      "        [-0.3804, -0.7333, -0.4196,  ...,  0.6863,  0.7255,  0.7333],\n",
      "        ...,\n",
      "        [-0.8510, -0.8431, -0.8353,  ..., -0.8588, -0.8588, -0.8588],\n",
      "        [-0.8510, -0.8275, -0.8196,  ..., -0.8510, -0.8431, -0.8196],\n",
      "        [-0.8353, -0.8902, -0.8980,  ..., -0.8118, -0.8353, -0.8118]],\n",
      "       device='cuda:0')\n",
      "tensor([[ 0.2549,  0.5529,  0.6471,  ...,  0.3333, -0.7569, -0.9922],\n",
      "        [ 0.0196, -0.0431,  0.0039,  ..., -0.0118, -0.4118, -0.9922],\n",
      "        [-0.0275,  0.0118,  0.1059,  ..., -0.1294, -0.2471, -0.9922],\n",
      "        ...,\n",
      "        [ 0.0588,  0.0431,  0.0902,  ..., -0.2627, -0.3333, -0.3569],\n",
      "        [ 0.0902,  0.1686,  0.0980,  ..., -0.1765, -0.2078, -0.2078],\n",
      "        [ 0.0275,  0.1137,  0.2000,  ...,  0.1059,  0.1608,  0.1451]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.9608,  0.6157, -0.6549,  ...,  0.6627,  0.6941,  0.7020],\n",
      "        [-0.9294, -0.3647,  0.7804,  ...,  0.6078,  0.6000,  0.6000],\n",
      "        [-0.9608, -0.2549, -0.9451,  ...,  0.5608,  0.4745,  0.4510],\n",
      "        ...,\n",
      "        [-0.6627, -0.7020, -0.7176,  ..., -0.8745, -0.8667, -0.8353],\n",
      "        [-0.7098, -0.7020, -0.6784,  ..., -0.9216, -0.9059, -0.8980],\n",
      "        [-0.7490, -0.7490, -0.7412,  ..., -0.8588, -0.8353, -0.8353]],\n",
      "       device='cuda:0')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/usr/local/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-0.4353, -0.1137, -0.6157,  ...,  0.4118,  0.3569, -1.0000],\n",
      "        [ 0.3725, -0.4275,  0.5529,  ...,  0.2157,  0.3098,  0.0431],\n",
      "        [-0.7725, -0.2157, -0.3333,  ...,  0.2863,  0.2235,  0.2627],\n",
      "        ...,\n",
      "        [-0.3804, -0.3333, -0.2706,  ..., -0.6078, -0.6314, -0.6078],\n",
      "        [-0.3255, -0.3412, -0.3098,  ..., -0.6314, -0.6549, -0.6627],\n",
      "        [-0.3412, -0.3098, -0.2706,  ..., -0.6392, -0.6706, -0.6627]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.2078, -0.7725,  0.2941,  ...,  0.7412,  0.5765,  0.5059],\n",
      "        [-0.9922, -0.8745,  0.5765,  ...,  0.5294,  0.4039,  0.3804],\n",
      "        [-0.3804, -0.5608,  0.4275,  ...,  0.1922,  0.1373,  0.1608],\n",
      "        ...,\n",
      "        [-0.7725, -0.7882, -0.8353,  ..., -0.8353, -0.8196, -0.8039],\n",
      "        [-0.7882, -0.7725, -0.8275,  ..., -0.8118, -0.7882, -0.7961],\n",
      "        [-0.7725, -0.7882, -0.8275,  ..., -0.8196, -0.8353, -0.8353]],\n",
      "       device='cuda:0')\n",
      "tensor([[ 0.4902, -1.0000, -0.7412,  ...,  0.5216,  0.4510,  0.3882],\n",
      "        [ 0.5843, -0.9765, -0.9843,  ...,  0.3569,  0.3255,  0.3647],\n",
      "        [ 0.5059, -0.9843, -0.3961,  ...,  0.4275,  0.3569,  0.3490],\n",
      "        ...,\n",
      "        [-0.8196, -0.7882, -0.8039,  ..., -0.7804, -0.7412, -0.7647],\n",
      "        [-0.8510, -0.8510, -0.8275,  ..., -0.8039, -0.8275, -0.8196],\n",
      "        [-0.7961, -0.7725, -0.7882,  ..., -0.8275, -0.8275, -0.8353]],\n",
      "       device='cuda:0')\n",
      "tensor([[ 0.1843, -0.6706, -0.2078,  ...,  0.8902,  0.8824,  0.6157],\n",
      "        [ 0.2078, -0.6863,  0.5529,  ...,  0.8275,  0.8196,  0.7176],\n",
      "        [ 0.1686, -0.6549,  0.4745,  ...,  0.5529,  0.5451,  0.5059],\n",
      "        ...,\n",
      "        [-0.7333, -0.7412, -0.7255,  ..., -0.9373, -0.9137, -0.9059],\n",
      "        [-0.8353, -0.8118, -0.8431,  ..., -0.9686, -0.9686, -0.9686],\n",
      "        [-0.9922, -1.0000, -0.9843,  ..., -0.9922, -0.9765, -0.9686]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.2471,  0.4980, -0.9294,  ...,  0.3647,  0.4118,  0.3647],\n",
      "        [-0.7882,  0.0588, -0.5765,  ...,  0.2078,  0.2000,  0.0980],\n",
      "        [-0.9608, -0.2863, -0.3647,  ...,  0.0824, -0.0118, -0.1216],\n",
      "        ...,\n",
      "        [-0.8275, -0.8275, -0.8353,  ..., -0.8588, -0.8431, -0.8431],\n",
      "        [-0.8353, -0.8588, -0.8431,  ..., -0.8353, -0.8196, -0.7961],\n",
      "        [-0.8745, -0.8510, -0.8275,  ..., -0.8039, -0.8196, -0.7961]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.9765, -0.2314, -0.1843,  ..., -0.1373, -0.2157, -0.2392],\n",
      "        [-1.0000,  0.1608,  0.3176,  ...,  0.1922,  0.1686,  0.1294],\n",
      "        [-0.7490, -0.3020, -0.2863,  ..., -0.3333, -0.3804, -0.4667],\n",
      "        ...,\n",
      "        [-0.8039, -0.7882, -0.7804,  ..., -0.7490, -0.7176, -0.7098],\n",
      "        [-0.7333, -0.7255, -0.7490,  ..., -0.6863, -0.6784, -0.6706],\n",
      "        [-0.7412, -0.7647, -0.7804,  ..., -0.7490, -0.7569, -0.7569]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.9686, -0.3098, -0.0039,  ...,  0.2471,  0.1922,  0.1843],\n",
      "        [-0.7098, -0.8824,  0.5451,  ...,  0.2471,  0.2784,  0.2471],\n",
      "        [-0.9843, -0.8510,  0.5373,  ...,  0.1451,  0.1765,  0.1529],\n",
      "        ...,\n",
      "        [-0.7412, -0.8275, -0.8745,  ..., -0.9765, -0.9529, -0.9765],\n",
      "        [-0.7490, -0.8353, -0.8667,  ..., -0.9686, -0.9843, -1.0000],\n",
      "        [-0.7961, -0.8275, -0.8275,  ..., -0.9608, -0.9608, -0.9608]],\n",
      "       device='cuda:0')\n",
      "tensor([[ 0.1922, -0.3490,  0.2549,  ..., -0.9922, -0.9922, -0.9922],\n",
      "        [-0.8980, -0.1765,  0.2941,  ..., -0.9922, -0.9922, -0.9922],\n",
      "        [ 0.4902, -0.8902,  0.2706,  ..., -0.9922, -0.9922, -0.9922],\n",
      "        ...,\n",
      "        [-0.8588, -0.8667, -0.8196,  ..., -0.5843, -0.5373, -0.5608],\n",
      "        [-0.8118, -0.8196, -0.8745,  ..., -0.5608, -0.5059, -0.4745],\n",
      "        [-0.8118, -0.8039, -0.8275,  ..., -0.6157, -0.6235, -0.6078]],\n",
      "       device='cuda:0')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/usr/local/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-0.9216,  0.4431, -0.8118,  ...,  0.0745,  0.1373, -0.0510],\n",
      "        [-0.8824, -0.1216, -0.5137,  ...,  0.2157,  0.2235,  0.1294],\n",
      "        [-0.9843, -0.1216, -0.5608,  ...,  0.0824,  0.1059,  0.1216],\n",
      "        ...,\n",
      "        [-0.8039, -0.8196, -0.8118,  ..., -0.8118, -0.8353, -0.8353],\n",
      "        [-0.8275, -0.8196, -0.7804,  ..., -0.8196, -0.8275, -0.8196],\n",
      "        [-0.7882, -0.8039, -0.8039,  ..., -0.8275, -0.8275, -0.8039]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.1686, -0.0353,  0.0667,  ...,  0.1922,  0.1686,  0.0510],\n",
      "        [-0.1059, -0.0667, -0.0510,  ...,  0.0275, -0.0824, -0.1922],\n",
      "        [-0.1922, -0.0196, -0.0275,  ..., -0.1059, -0.1529, -0.1216],\n",
      "        ...,\n",
      "        [-0.4588, -0.4431, -0.3647,  ..., -0.1451, -0.0824, -0.0118],\n",
      "        [-0.5216, -0.4980, -0.4667,  ..., -0.0980, -0.0039, -0.0039],\n",
      "        [-0.6078, -0.5765, -0.5294,  ..., -0.0902, -0.0902, -0.1294]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.3961, -0.4118, -0.4118,  ..., -0.9922, -0.9922, -0.9922],\n",
      "        [-0.2863, -0.3490, -0.3804,  ..., -0.9922, -0.9922, -0.9922],\n",
      "        [-0.2549, -0.2471, -0.2157,  ..., -0.9922, -0.9922, -0.9922],\n",
      "        ...,\n",
      "        [-0.7725, -0.7804, -0.8118,  ..., -0.6941, -0.6784, -0.6392],\n",
      "        [-0.7804, -0.7569, -0.7804,  ..., -0.6235, -0.6627, -0.6392],\n",
      "        [-0.6941, -0.7490, -0.7882,  ..., -0.6314, -0.6157, -0.6157]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.9843, -0.9608,  0.4667,  ...,  0.2706,  0.1529, -0.9922],\n",
      "        [-0.9843, -0.9843,  0.3333,  ...,  0.1373,  0.0118, -0.9765],\n",
      "        [-0.9843, -0.4118,  0.4980,  ...,  0.2235,  0.0667, -0.2392],\n",
      "        ...,\n",
      "        [-0.6941, -0.6706, -0.6549,  ..., -0.6392, -0.6314, -0.6471],\n",
      "        [-0.7569, -0.7098, -0.7098,  ..., -0.6392, -0.6784, -0.6863],\n",
      "        [-0.7569, -0.7490, -0.7333,  ..., -0.6706, -0.6863, -0.7020]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.9529,  0.5294,  0.4431,  ...,  0.7412,  0.6784,  0.5843],\n",
      "        [-0.9451,  0.4431,  0.5373,  ...,  0.6784,  0.6627,  0.6000],\n",
      "        [-0.9373,  0.4588, -0.3725,  ...,  0.5529,  0.4902,  0.4196],\n",
      "        ...,\n",
      "        [-0.6157, -0.6314, -0.6235,  ..., -0.6784, -0.6549, -0.6157],\n",
      "        [-0.6157, -0.6706, -0.7176,  ..., -0.6627, -0.6549, -0.6157],\n",
      "        [-0.6549, -0.6627, -0.7020,  ..., -0.7098, -0.6863, -0.7020]],\n",
      "       device='cuda:0')\n",
      "tensor([[ 0.2314, -0.9922,  0.2627,  ...,  1.0000,  0.9608,  0.9216],\n",
      "        [-0.7961, -0.2471,  0.5843,  ...,  0.8275,  0.8745,  0.8353],\n",
      "        [-0.9216, -0.9608,  0.5608,  ...,  0.7020,  0.6706,  0.7098],\n",
      "        ...,\n",
      "        [-0.6392, -0.6784, -0.6863,  ..., -0.7804, -0.7569, -0.7569],\n",
      "        [-0.7333, -0.7176, -0.6627,  ..., -0.7020, -0.7176, -0.7333],\n",
      "        [-0.7490, -0.7333, -0.7098,  ..., -0.7569, -0.7020, -0.6941]],\n",
      "       device='cuda:0')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "../utils.py:265: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).\n",
      "  fig = plt.figure(frameon=False)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-0.9922, -0.9765,  0.2784,  ...,  0.3569,  0.3333,  0.3725],\n",
      "        [-0.9843, -0.4431,  0.4510,  ...,  0.3961,  0.3647,  0.3255],\n",
      "        [ 0.1922,  0.5451,  0.4745,  ...,  0.3255,  0.3098,  0.2392],\n",
      "        ...,\n",
      "        [-0.8353, -0.8588, -0.8667,  ..., -0.8275, -0.7882, -0.8118],\n",
      "        [-0.8353, -0.7961, -0.7961,  ..., -0.7961, -0.8275, -0.8431],\n",
      "        [-0.7961, -0.8196, -0.8275,  ..., -0.7961, -0.8353, -0.8431]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.9922, -0.9922, -0.0510,  ..., -0.9922, -0.9922, -0.9922],\n",
      "        [ 0.4902, -0.3882, -0.0980,  ..., -0.9922, -0.9922, -0.9922],\n",
      "        [-0.6235,  0.4667, -0.9765,  ..., -0.9922, -0.9922, -0.9922],\n",
      "        ...,\n",
      "        [-0.8039, -0.8039, -0.8431,  ..., -0.7333, -0.7098, -0.6706],\n",
      "        [-0.8196, -0.7882, -0.7725,  ..., -0.6235, -0.6000, -0.6392],\n",
      "        [-0.8275, -0.8118, -0.7961,  ..., -0.6549, -0.7020, -0.7569]],\n",
      "       device='cuda:0')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/usr/local/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-0.9843, -0.9843, -0.9922,  ..., -0.0275, -0.0745, -0.0980],\n",
      "        [ 0.0980, -0.8902, -0.9922,  ..., -0.0196,  0.0431, -0.0039],\n",
      "        [ 0.5216, -0.5451, -0.8980,  ...,  0.0745,  0.0667,  0.0196],\n",
      "        ...,\n",
      "        [-0.7569, -0.7569, -0.7569,  ..., -0.7647, -0.7412, -0.7333],\n",
      "        [-0.7569, -0.7725, -0.7647,  ..., -0.6941, -0.6706, -0.6627],\n",
      "        [-0.7882, -0.7961, -0.7882,  ..., -0.6863, -0.7176, -0.7412]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.8588,  0.0118,  0.4431,  ...,  0.4510,  0.5373,  0.5686],\n",
      "        [-0.2941,  0.0745, -0.7725,  ...,  0.2941,  0.3804,  0.2471],\n",
      "        [ 0.0588, -0.5765, -0.7490,  ...,  0.1529,  0.0667,  0.0353],\n",
      "        ...,\n",
      "        [-0.8588, -0.8980, -0.9216,  ..., -0.8824, -0.8667, -0.8824],\n",
      "        [-0.8510, -0.8902, -0.9137,  ..., -0.8745, -0.8588, -0.8667],\n",
      "        [-0.8431, -0.8824, -0.9137,  ..., -0.8902, -0.8745, -0.8824]],\n",
      "       device='cuda:0')\n",
      "tensor([[ 0.2471, -0.7804, -0.7098,  ...,  0.6941,  0.5843,  0.5059],\n",
      "        [ 0.1294, -0.7882, -0.7020,  ...,  0.4980,  0.4353,  0.4039],\n",
      "        [-0.2314, -0.7804, -0.7020,  ...,  0.2235,  0.1843,  0.1529],\n",
      "        ...,\n",
      "        [-0.0196,  0.0118,  0.0196,  ..., -0.6863, -0.7098, -0.7176],\n",
      "        [ 0.1137,  0.2392,  0.2471,  ..., -0.6784, -0.6627, -0.6549],\n",
      "        [ 0.0353,  0.1843,  0.2706,  ..., -0.6784, -0.6706, -0.6627]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.0118,  0.1686, -0.9922,  ...,  0.9451,  0.7725, -0.6157],\n",
      "        [-0.4353, -0.4588, -0.6549,  ...,  0.8588,  0.7176,  0.5059],\n",
      "        [-0.9843, -0.9765, -0.9843,  ...,  0.7098,  0.7020,  0.5686],\n",
      "        ...,\n",
      "        [-0.7490, -0.7490, -0.6941,  ..., -0.8824, -0.8588, -0.8039],\n",
      "        [-0.7176, -0.7333, -0.7255,  ..., -0.8353, -0.8039, -0.7961],\n",
      "        [-0.6235, -0.5529, -0.5686,  ..., -0.8745, -0.8588, -0.8353]],\n",
      "       device='cuda:0')\n",
      "tensor([[ 0.5686, -0.0510, -0.9765,  ...,  0.6392,  0.6314,  0.6157],\n",
      "        [-0.9765,  0.4196, -0.7569,  ...,  0.6000,  0.5451,  0.4980],\n",
      "        [-0.8902, -0.2078, -0.4588,  ...,  0.4118,  0.3333,  0.3412],\n",
      "        ...,\n",
      "        [-0.7725, -0.7804, -0.8196,  ..., -0.8196, -0.8118, -0.8196],\n",
      "        [-0.7961, -0.8118, -0.8275,  ..., -0.8039, -0.8039, -0.8275],\n",
      "        [-0.8275, -0.8039, -0.7647,  ..., -0.7647, -0.7569, -0.7804]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.4745, -0.4039, -0.3961,  ..., -0.4980, -0.5529, -0.5451],\n",
      "        [-0.8275, -0.7490, -0.7098,  ..., -0.5686, -0.5451, -0.5451],\n",
      "        [-0.8431, -0.8118, -0.7647,  ..., -0.5608, -0.6000, -0.6471],\n",
      "        ...,\n",
      "        [-0.9529, -0.9529, -0.9373,  ..., -0.8667, -0.8667, -0.8588],\n",
      "        [-0.8824, -0.8980, -0.8824,  ..., -0.8588, -0.8510, -0.8353],\n",
      "        [-0.9216, -0.9137, -0.9137,  ..., -0.8275, -0.8275, -0.8353]],\n",
      "       device='cuda:0')\n",
      "tensor([[-1.0000, -0.9922, -0.9765,  ..., -0.9922, -0.9922, -0.9922],\n",
      "        [-0.6627, -0.9529, -0.2706,  ..., -0.9922, -0.9922, -0.9922],\n",
      "        [-0.4824, -0.2078,  0.1451,  ..., -0.9922, -0.9922, -0.9922],\n",
      "        ...,\n",
      "        [-0.7804, -0.7490, -0.7804,  ..., -0.7961, -0.7961, -0.7412],\n",
      "        [-0.8588, -0.8275, -0.8353,  ..., -0.8275, -0.7882, -0.7804],\n",
      "        [-0.8667, -0.8745, -0.8431,  ..., -0.7255, -0.7255, -0.7255]],\n",
      "       device='cuda:0')\n",
      "tensor([[ 0.2549,  0.1059,  0.5922,  ...,  0.6000,  0.6314,  0.5922],\n",
      "        [-0.9294,  0.3412,  0.5686,  ...,  0.5608,  0.5059,  0.4824],\n",
      "        [ 0.5137, -0.9216,  0.6235,  ...,  0.4980,  0.3804,  0.2863],\n",
      "        ...,\n",
      "        [-0.8353, -0.7882, -0.7412,  ..., -0.6863, -0.6784, -0.6941],\n",
      "        [-0.7569, -0.7804, -0.7961,  ..., -0.7176, -0.7020, -0.6941],\n",
      "        [-0.8275, -0.8039, -0.8275,  ..., -0.6863, -0.6392, -0.6392]],\n",
      "       device='cuda:0')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/usr/local/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 0.5765, -0.2706,  0.8353,  ...,  0.7725,  0.7725,  0.7961],\n",
      "        [ 0.3098,  0.4353,  0.7412,  ...,  0.7098,  0.6941,  0.7176],\n",
      "        [-0.9216, -0.3647,  0.6235,  ...,  0.6235,  0.5686,  0.5843],\n",
      "        ...,\n",
      "        [-0.8431, -0.8510, -0.8824,  ..., -0.8431, -0.7333, -0.7020],\n",
      "        [-0.8745, -0.9137, -0.9137,  ..., -0.8118, -0.7412, -0.7176],\n",
      "        [-0.8196, -0.8118, -0.8275,  ..., -0.7961, -0.7333, -0.7176]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.9529,  0.7020,  0.6784,  ...,  0.5294,  0.4510,  0.3961],\n",
      "        [-0.6627,  0.6863,  0.6784,  ...,  0.4353,  0.4510,  0.4902],\n",
      "        [-0.4824,  0.5216,  0.6549,  ...,  0.4196,  0.3882,  0.3333],\n",
      "        ...,\n",
      "        [-0.8196, -0.7961, -0.7961,  ..., -0.8275, -0.8353, -0.8431],\n",
      "        [-0.8667, -0.8353, -0.8275,  ..., -0.8431, -0.8510, -0.8431],\n",
      "        [-0.8510, -0.8588, -0.8510,  ..., -0.7961, -0.8196, -0.8275]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.9922, -0.9765, -0.0275,  ...,  0.7020,  0.7412,  0.8275],\n",
      "        [ 0.5216, -0.5922, -0.4745,  ...,  0.6549,  0.6706,  0.7020],\n",
      "        [ 0.3569,  0.1059, -0.9843,  ...,  0.5686,  0.6078,  0.5059],\n",
      "        ...,\n",
      "        [-0.4902, -0.5137, -0.5216,  ..., -0.8510, -0.8118, -0.8588],\n",
      "        [-0.5608, -0.5294, -0.5451,  ..., -0.9059, -0.8431, -0.8118],\n",
      "        [-0.9922, -0.9843, -0.8431,  ..., -0.9686, -0.9608, -0.9686]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.9137,  0.5216, -0.8431,  ...,  0.4667,  0.4824, -0.4118],\n",
      "        [-0.8824,  0.3255, -0.9765,  ...,  0.4431,  0.4196,  0.1373],\n",
      "        [-0.8745,  0.3333, -0.9529,  ...,  0.3882,  0.3098,  0.1137],\n",
      "        ...,\n",
      "        [-0.6941, -0.6863, -0.7098,  ..., -0.4667, -0.4431, -0.4431],\n",
      "        [-0.7333, -0.7098, -0.6706,  ..., -0.4275, -0.4824, -0.5137],\n",
      "        [-0.6627, -0.6941, -0.7176,  ..., -0.4667, -0.5294, -0.5608]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.0196,  0.4118,  0.6863,  ...,  0.4902,  0.0196, -0.9765],\n",
      "        [-0.0902, -0.0824,  0.0667,  ...,  0.0824, -0.0667, -0.9451],\n",
      "        [-0.1451, -0.1059,  0.0745,  ..., -0.1608, -0.1686, -0.9843],\n",
      "        ...,\n",
      "        [-0.4980, -0.5137, -0.4902,  ..., -0.3020, -0.3098, -0.3020],\n",
      "        [-0.5059, -0.4980, -0.4667,  ..., -0.3882, -0.3490, -0.3333],\n",
      "        [-0.5137, -0.5373, -0.4980,  ..., -0.3412, -0.3098, -0.3412]],\n",
      "       device='cuda:0')\n",
      "tensor([[ 0.5137,  0.6235,  0.6314,  ...,  0.5294,  0.5137,  0.2941],\n",
      "        [ 0.1686,  0.4902,  0.5059,  ...,  0.3961,  0.4118,  0.2157],\n",
      "        [ 0.0196,  0.1843,  0.1686,  ...,  0.1059,  0.1451,  0.0353],\n",
      "        ...,\n",
      "        [-0.1922, -0.2000, -0.2314,  ..., -0.7804, -0.7725, -0.7255],\n",
      "        [-0.2784, -0.2627, -0.2392,  ..., -0.7333, -0.7569, -0.7490],\n",
      "        [-0.3255, -0.2941, -0.2392,  ..., -0.7569, -0.7098, -0.7098]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.9686,  0.6863,  0.9451,  ...,  0.4824,  0.5294,  0.5294],\n",
      "        [-0.1451,  0.4196,  0.7961,  ...,  0.4667,  0.5529,  0.4980],\n",
      "        [-0.0275,  0.2549,  0.3725,  ...,  0.5922,  0.5529,  0.4510],\n",
      "        ...,\n",
      "        [-0.7569, -0.7255, -0.7176,  ..., -0.7804, -0.7647, -0.7255],\n",
      "        [-0.7961, -0.7569, -0.7098,  ..., -0.7804, -0.7569, -0.7333],\n",
      "        [-0.7725, -0.7569, -0.7098,  ..., -0.7725, -0.7490, -0.7569]],\n",
      "       device='cuda:0')\n",
      "tensor([[ 0.0196,  0.0196,  0.2549,  ...,  0.5216,  0.5922,  0.6314],\n",
      "        [-0.8824, -0.7490,  0.2157,  ...,  0.4667,  0.5608,  0.5608],\n",
      "        [-0.9294, -0.9608,  0.1137,  ...,  0.6078,  0.6314,  0.5294],\n",
      "        ...,\n",
      "        [-0.7647, -0.7804, -0.7961,  ..., -0.7725, -0.7961, -0.7882],\n",
      "        [-0.7725, -0.7961, -0.7647,  ..., -0.7725, -0.7804, -0.8118],\n",
      "        [-0.7882, -0.7725, -0.7647,  ..., -0.7882, -0.7725, -0.7804]],\n",
      "       device='cuda:0')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/usr/local/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-0.6863,  0.4353,  0.5922,  ...,  0.4510,  0.4824,  0.4745],\n",
      "        [ 0.4039, -0.6941, -0.9529,  ...,  0.4039,  0.3412,  0.3176],\n",
      "        [ 0.5686, -0.9843, -0.9294,  ...,  0.2392,  0.1686,  0.1922],\n",
      "        ...,\n",
      "        [-0.8824, -0.8824, -0.8431,  ..., -0.4275, -0.3490, -0.2706],\n",
      "        [-0.8902, -0.8980, -0.8667,  ..., -0.5294, -0.4745, -0.4431],\n",
      "        [-0.8745, -0.8745, -0.8667,  ..., -0.6471, -0.6314, -0.6314]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.0588, -0.0745,  0.3176,  ...,  0.5451,  0.5608,  0.4667],\n",
      "        [-0.9922, -0.9843,  0.3882,  ...,  0.3804,  0.3098,  0.2392],\n",
      "        [-0.9843, -0.9922,  0.3098,  ...,  0.3725,  0.3725,  0.3412],\n",
      "        ...,\n",
      "        [-0.8118, -0.8431, -0.8431,  ..., -0.8353, -0.8431, -0.8353],\n",
      "        [-0.7882, -0.7490, -0.7725,  ..., -0.8353, -0.8353, -0.8588],\n",
      "        [-0.7961, -0.8118, -0.8431,  ..., -0.8196, -0.8353, -0.8196]],\n",
      "       device='cuda:0')\n",
      "tensor([[ 0.5843,  0.3882, -1.0000,  ..., -0.9922, -0.9922, -0.9922],\n",
      "        [-0.9686, -0.5843, -0.2549,  ..., -0.9922, -0.9922, -0.9922],\n",
      "        [-1.0000, -0.9608,  0.5216,  ..., -0.9922, -0.9922, -0.9922],\n",
      "        ...,\n",
      "        [-0.9922, -0.9922, -0.9843,  ..., -0.9922, -0.9922, -0.9922],\n",
      "        [-0.9922, -0.9922, -0.9843,  ..., -0.9922, -0.9922, -0.9922],\n",
      "        [-0.9843, -0.9843, -0.9843,  ..., -0.9922, -0.9922, -0.9922]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.8510,  0.5529, -0.2627,  ...,  0.6392,  0.6471,  0.6235],\n",
      "        [ 0.0118, -0.2392, -0.2549,  ...,  0.6078,  0.6549,  0.6157],\n",
      "        [ 0.2157, -0.5765, -0.9765,  ...,  0.5529,  0.5137,  0.5451],\n",
      "        ...,\n",
      "        [-0.8588, -0.8275, -0.8353,  ..., -0.8196, -0.8275, -0.8588],\n",
      "        [-0.8275, -0.8039, -0.7961,  ..., -0.8980, -0.8588, -0.8745],\n",
      "        [-0.8196, -0.8353, -0.8431,  ..., -0.8353, -0.8510, -0.8745]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.4118, -0.1765,  0.5922,  ...,  0.5608, -0.9922, -0.9922],\n",
      "        [-0.9922, -0.9922,  0.5843,  ...,  0.4980,  0.2784, -0.9843],\n",
      "        [ 0.5922,  0.3725,  0.5686,  ...,  0.3569,  0.3725, -0.9608],\n",
      "        ...,\n",
      "        [-0.8039, -0.7490, -0.7255,  ..., -0.7647, -0.7490, -0.7412],\n",
      "        [-0.7333, -0.7333, -0.7176,  ..., -0.7333, -0.7333, -0.7412],\n",
      "        [-0.7176, -0.8039, -0.7804,  ..., -0.6706, -0.6471, -0.6784]],\n",
      "       device='cuda:0')\n",
      "tensor([[ 0.3412, -0.0196, -0.5686,  ...,  0.7412,  0.7804,  0.7490],\n",
      "        [ 0.3333,  0.5765, -0.4275,  ...,  0.7098,  0.7098,  0.5373],\n",
      "        [ 0.3333,  0.5686, -0.9765,  ...,  0.5294,  0.5137,  0.5137],\n",
      "        ...,\n",
      "        [-0.6627, -0.6706, -0.6784,  ..., -0.9608, -0.9686, -0.9529],\n",
      "        [-0.6863, -0.6706, -0.6941,  ..., -0.8510, -0.8667, -0.8902],\n",
      "        [-0.6235, -0.5922, -0.6471,  ..., -0.8039, -0.8118, -0.8667]],\n",
      "       device='cuda:0')\n",
      "tensor([[ 0.5765, -0.7804,  0.0039,  ...,  0.7647,  0.7490,  0.8039],\n",
      "        [-0.9765, -0.7569,  0.0039,  ...,  0.6784,  0.6863,  0.7412],\n",
      "        [ 0.5922, -0.7647, -0.0196,  ...,  0.6549,  0.6314,  0.5529],\n",
      "        ...,\n",
      "        [-0.7098, -0.7098, -0.7412,  ..., -0.7725, -0.8039, -0.8118],\n",
      "        [-0.7961, -0.7647, -0.7569,  ..., -0.7882, -0.7490, -0.7647],\n",
      "        [-0.8118, -0.8353, -0.8510,  ..., -0.7412, -0.7333, -0.7725]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.7333, -0.2941,  0.6000,  ...,  0.7490,  0.7333,  0.6941],\n",
      "        [-0.7020, -0.2941,  0.1922,  ...,  0.6000,  0.6314,  0.5843],\n",
      "        [-0.6471, -0.3020, -0.9529,  ...,  0.3725,  0.3490,  0.4353],\n",
      "        ...,\n",
      "        [-0.8196, -0.8039, -0.8275,  ..., -0.7961, -0.8118, -0.8196],\n",
      "        [-0.7647, -0.7725, -0.7725,  ..., -0.7490, -0.7725, -0.7647],\n",
      "        [-0.7804, -0.7647, -0.7569,  ..., -0.7647, -0.8196, -0.8196]],\n",
      "       device='cuda:0')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/usr/local/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-0.6941, -0.9843, -0.5765,  ...,  0.8118,  0.7961,  0.6627],\n",
      "        [ 0.5451, -0.0039,  0.0196,  ...,  0.7804,  0.7569,  0.6471],\n",
      "        [-0.3882, -0.3176,  0.5922,  ...,  0.6314,  0.6157,  0.6392],\n",
      "        ...,\n",
      "        [-0.8902, -0.8902, -0.9059,  ..., -0.8196, -0.8275, -0.8118],\n",
      "        [-0.8353, -0.8118, -0.8510,  ..., -0.7333, -0.7882, -0.8039],\n",
      "        [-0.8196, -0.8667, -0.8745,  ..., -0.7804, -0.8039, -0.8039]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.9922, -0.9922, -0.9765,  ...,  0.7255,  0.7725,  0.7255],\n",
      "        [-0.9843,  0.5059,  0.1686,  ...,  0.8118,  0.8510,  0.7255],\n",
      "        [-0.6471, -0.5686, -0.9686,  ...,  0.7804,  0.6706,  0.5529],\n",
      "        ...,\n",
      "        [-0.8588, -0.8275, -0.7961,  ..., -0.6941, -0.6627, -0.7020],\n",
      "        [-0.8824, -0.8431, -0.7647,  ..., -0.7255, -0.7412, -0.7882],\n",
      "        [-0.8118, -0.8039, -0.7333,  ..., -0.7725, -0.7569, -0.7569]],\n",
      "       device='cuda:0')\n",
      "tensor([[ 0.2549,  0.1922,  0.1608,  ...,  0.2314,  0.2157,  0.2549],\n",
      "        [ 0.1294,  0.1608,  0.1765,  ...,  0.1529,  0.1765,  0.1451],\n",
      "        [ 0.3412,  0.3412,  0.2471,  ...,  0.2000,  0.1294,  0.1608],\n",
      "        ...,\n",
      "        [-0.8431, -0.8510, -0.8275,  ..., -0.9373, -0.8667, -0.8588],\n",
      "        [-0.8196, -0.8275, -0.8510,  ..., -0.8353, -0.8667, -0.9451],\n",
      "        [-0.7961, -0.8196, -0.8902,  ..., -0.8902, -0.9059, -0.8980]],\n",
      "       device='cuda:0')\n",
      "tensor([[ 0.4588,  0.5294,  0.5686,  ...,  0.3647,  0.3569,  0.3647],\n",
      "        [ 0.4275,  0.4275,  0.4118,  ...,  0.4039,  0.3725,  0.3412],\n",
      "        [ 0.3882,  0.3882,  0.3882,  ...,  0.3882,  0.3882,  0.3647],\n",
      "        ...,\n",
      "        [-0.8431, -0.8353, -0.8275,  ..., -0.8667, -0.8510, -0.8431],\n",
      "        [-0.8667, -0.8353, -0.8196,  ..., -0.8431, -0.8353, -0.8353],\n",
      "        [-0.8980, -0.8667, -0.8431,  ..., -0.8510, -0.8431, -0.8353]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.9608, -0.0902, -0.0431,  ..., -0.0824, -0.0902, -0.1137],\n",
      "        [-0.8667, -0.0118, -0.0196,  ...,  0.0118,  0.0196,  0.0039],\n",
      "        [-0.3647, -0.4588, -0.5059,  ..., -0.4275, -0.4118, -0.4275],\n",
      "        ...,\n",
      "        [-0.8196, -0.8275, -0.7961,  ..., -0.6000, -0.6627, -0.6863],\n",
      "        [-0.7725, -0.7647, -0.7804,  ..., -0.5922, -0.6157, -0.5922],\n",
      "        [-0.6627, -0.6784, -0.7020,  ..., -0.5686, -0.5843, -0.5608]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.6235,  0.5137, -0.6471,  ...,  0.2078, -0.4824, -0.9765],\n",
      "        [-0.2235,  0.1686, -0.9765,  ...,  0.0980,  0.0118, -0.9922],\n",
      "        [-0.2314,  0.1843, -0.9843,  ...,  0.0431, -0.0039, -0.4902],\n",
      "        ...,\n",
      "        [-0.7098, -0.7098, -0.6941,  ..., -0.5529, -0.5686, -0.5765],\n",
      "        [-0.7020, -0.6863, -0.6863,  ..., -0.6157, -0.6078, -0.6000],\n",
      "        [-0.7333, -0.7176, -0.6784,  ..., -0.6627, -0.6392, -0.6314]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.9451,  0.6314,  0.7020,  ...,  0.4510,  0.4745,  0.5059],\n",
      "        [-0.6314,  0.5686,  0.6549,  ...,  0.4980,  0.4510,  0.3647],\n",
      "        [-0.4588,  0.6000,  0.6314,  ...,  0.6078,  0.6078,  0.4353],\n",
      "        ...,\n",
      "        [-0.7725, -0.7490, -0.7333,  ..., -0.6863, -0.7098, -0.7255],\n",
      "        [-0.7647, -0.7725, -0.7647,  ..., -0.6784, -0.6549, -0.6392],\n",
      "        [-0.7569, -0.7804, -0.8039,  ..., -0.6549, -0.6392, -0.6784]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.1137, -0.4196, -0.9765,  ...,  0.8824,  0.7412,  0.2627],\n",
      "        [-0.4824,  0.3569, -0.4431,  ...,  0.9922,  0.7490,  0.4745],\n",
      "        [-0.1686, -0.7882,  0.3569,  ...,  0.7098,  0.5922,  0.5059],\n",
      "        ...,\n",
      "        [-0.8196, -0.8275, -0.8353,  ..., -0.7333, -0.7647, -0.8118],\n",
      "        [-0.8118, -0.7804, -0.7725,  ..., -0.7490, -0.7804, -0.8118],\n",
      "        [-0.8588, -0.8353, -0.8431,  ..., -0.8902, -0.8902, -0.8902]],\n",
      "       device='cuda:0')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/usr/local/lib/python3.7/site-packages/sklearn/metrics/_classification.py:1318: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-0.9843, -0.6000, -0.0745,  ...,  0.7176,  0.6941,  0.5608],\n",
      "        [-0.7647,  0.4980, -0.4824,  ...,  0.5843,  0.5843,  0.4275],\n",
      "        [ 0.0275, -0.3804, -0.0039,  ...,  0.4275,  0.3804,  0.4431],\n",
      "        ...,\n",
      "        [-0.7961, -0.7725, -0.7569,  ..., -0.8353, -0.8196, -0.8196],\n",
      "        [-0.8353, -0.8275, -0.7961,  ..., -0.7961, -0.7961, -0.8510],\n",
      "        [-0.8745, -0.8431, -0.8745,  ..., -0.8745, -0.8431, -0.8353]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.0039,  0.0667,  0.1529,  ...,  0.0745,  0.0667,  0.0196],\n",
      "        [-0.2235, -0.0667,  0.0431,  ...,  0.1294,  0.1059,  0.0353],\n",
      "        [-0.2784, -0.1608, -0.0510,  ...,  0.0353,  0.0196, -0.0431],\n",
      "        ...,\n",
      "        [-0.9059, -0.9137, -0.9137,  ..., -0.8902, -0.8824, -0.9137],\n",
      "        [-0.9137, -0.9294, -0.9373,  ..., -0.8667, -0.8588, -0.8588],\n",
      "        [-0.9216, -0.9294, -0.9373,  ..., -0.9294, -0.9294, -0.9137]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.9451,  0.8118,  0.7412,  ...,  0.8980,  0.7961,  0.7569],\n",
      "        [-0.6549,  0.7725,  0.7804,  ...,  0.8980,  0.8431,  0.7412],\n",
      "        [-0.4510,  0.6157,  0.6471,  ...,  0.7882,  0.7412,  0.7490],\n",
      "        ...,\n",
      "        [-0.8039, -0.8118, -0.8196,  ..., -0.7255, -0.7569, -0.7569],\n",
      "        [-0.8745, -0.8667, -0.8510,  ..., -0.7647, -0.7647, -0.7804],\n",
      "        [-0.8353, -0.8118, -0.7961,  ..., -0.7725, -0.7647, -0.7647]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.9216,  0.4510, -0.7882,  ...,  0.3098,  0.3176,  0.2863],\n",
      "        [-0.8510, -0.1765, -0.4824,  ...,  0.2863,  0.2235,  0.1686],\n",
      "        [-0.9922, -0.1137, -0.4667,  ...,  0.2392,  0.2314,  0.1451],\n",
      "        ...,\n",
      "        [-0.2784, -0.2471, -0.2941,  ..., -0.5922, -0.6000, -0.6078],\n",
      "        [-0.2706, -0.2235, -0.2157,  ..., -0.5686, -0.6000, -0.6314],\n",
      "        [-0.3255, -0.2863, -0.2471,  ..., -0.5843, -0.5765, -0.5765]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.1294, -0.1373, -0.1059,  ...,  0.0196, -0.0275,  0.0431],\n",
      "        [-0.1843, -0.2314, -0.2235,  ...,  0.0667,  0.1137,  0.1686],\n",
      "        [-0.1843, -0.1922, -0.1529,  ...,  0.0667,  0.0824,  0.0588],\n",
      "        ...,\n",
      "        [-0.5608, -0.5843, -0.5843,  ..., -0.6392, -0.7098, -0.7490],\n",
      "        [-0.5373, -0.5686, -0.6000,  ..., -0.6157, -0.6863, -0.7569],\n",
      "        [-0.4588, -0.4824, -0.5294,  ..., -0.7412, -0.7490, -0.7333]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.9686,  0.6000,  0.4353,  ...,  0.6941,  0.6392,  0.4353],\n",
      "        [-0.0039,  0.5843,  0.5294,  ...,  0.5608,  0.5137,  0.3882],\n",
      "        [-0.0039,  0.5765, -0.3725,  ...,  0.4902,  0.4667,  0.3490],\n",
      "        ...,\n",
      "        [-0.8431, -0.8745, -0.9294,  ..., -0.8902, -0.8431, -0.7961],\n",
      "        [-0.8196, -0.8588, -0.8353,  ..., -0.8902, -0.8824, -0.8510],\n",
      "        [-0.8745, -0.9059, -0.8980,  ..., -0.8745, -0.8980, -0.9059]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.8353,  0.3098, -1.0000,  ...,  0.8196,  0.6863,  0.6000],\n",
      "        [-0.8980,  0.4275, -0.9843,  ...,  0.7647,  0.6314,  0.4039],\n",
      "        [-0.9137,  0.1608,  0.1451,  ...,  0.6549,  0.4039,  0.1137],\n",
      "        ...,\n",
      "        [-0.8275, -0.8510, -0.8353,  ..., -0.8510, -0.8588, -0.8510],\n",
      "        [-0.8353, -0.8588, -0.8353,  ..., -0.8275, -0.8196, -0.8118],\n",
      "        [-0.8510, -0.8510, -0.8275,  ..., -0.8510, -0.8275, -0.8275]],\n",
      "       device='cuda:0')\n",
      "tensor([[-0.2000,  0.5765,  0.5529,  ...,  0.8039,  0.8196,  0.8510],\n",
      "        [ 0.0039,  0.5765, -0.1529,  ...,  0.8039,  0.7804,  0.7961],\n",
      "        [-0.9294,  0.6000, -0.9686,  ...,  0.6000,  0.6392,  0.6157],\n",
      "        ...,\n",
      "        [-0.7569, -0.7490, -0.7647,  ..., -0.8902, -0.8980, -0.8824],\n",
      "        [-0.7804, -0.8118, -0.8039,  ..., -0.8196, -0.8667, -0.8902],\n",
      "        [-0.7176, -0.7333, -0.7725,  ..., -0.8902, -0.8510, -0.8039]],\n",
      "       device='cuda:0')\n",
      "\n",
      "\n",
      "[VAL RESULT]\n",
      "\n",
      "acc_zeropadded:0.7188 avg_acc:0.5938 avg_acc_fixed:0.5781\n",
      "precision_macro_zeropadded:0.3594 precision_macro_approx:0.5884 precision_macro_approx_fixed:0.5905\n",
      "precision_micro_zeropadded:0.7188 precision_micro_approx:0.5938 precision_micro_approx_fixed:0.5781\n",
      "recall_macro_zeropadded:0.5000 recall_macro_approx:0.5708 recall_macro_approx_fixed:0.5723\n",
      "recall_micro_zeropadded:0.7188 recall_micro_approx:0.5938 recall_micro_approx_fixed:0.5781\n",
      "f1_macro_zeropadded:0.4161 f1_macro_approx:0.5419 f1_macro_approx_fixed:0.5335\n",
      "f1_micro_zeropadded:0.7188 f1_micro_approx:0.5781 f1_micro_approx_fixed:0.5781\n",
      "vmi:1.0864 vmi_fixed:1.0725 vmi_zeropadded:-7.0383\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import argparse\n",
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]='3'\n",
    "import sys\n",
    "import copy\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib as mpl\n",
    "sys.path.append('../')\n",
    "import numpy as np\n",
    "from return_data import return_data\n",
    "#from scipy import misc\n",
    "#import cv2\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.autograd import Variable\n",
    "from utils import str2bool, label2binary, cuda, idxtobool, UnknownDatasetError, UnknownModelError, index_transfer, save_batch, save_batch2\n",
    "from pathlib import Path\n",
    "from torch.nn import functional as F\n",
    "from sklearn.preprocessing import label_binarize\n",
    "from sklearn.metrics import f1_score, precision_score, recall_score, roc_auc_score\n",
    "import matplotlib\n",
    "matplotlib.use('agg')\n",
    "import matplotlib.pyplot as plt\n",
    "matplotlib.pyplot.switch_backend('agg')\n",
    "##https://github.com/lightdogs/pytorch-smoothgrad\n",
    "from lib.gradients import VanillaGrad, SmoothGrad #, GuidedBackpropGrad, GuidedBackpropSmoothGrad\n",
    "from lib.image_utils import Segmentation\n",
    "from lib.labels import IMAGENET_LABELS\n",
    "\n",
    "#%%    \n",
    "def parse_args():\n",
    "\n",
    "    parser = argparse.ArgumentParser()\n",
    "    \n",
    "    parser.add_argument('--dataset', default='mnist', type = str, help='dataset name: imdb, imdb, mnist')\n",
    "    parser.add_argument('--default_dir', default='.', type = str, help='default directory path')\n",
    "    parser.add_argument('--data_dir', default='\"../mnist/dataset/Dataset_BUSI_AN/train/images\"', type = str, help='data directory path')\n",
    "    parser.add_argument('--method', default='smoothgrad', type=str, help = 'interpretable ML method: saliency, taylor')\n",
    "    parser.add_argument('--batch_size', type=int, default=8, metavar='N', help='input batch size for training')\n",
    "    parser.add_argument('--model_name', default='original_BUSI7.ckpt', type=str, help = 'if train is True, model name to be saved, otherwise model name to be loaded')\n",
    "    #parser.add_argument('--chunk_size', default = 1, type = int, help='chunk size. for image, chunk x chunk will be the actual chunk size')\n",
    "    parser.add_argument('--chunk_size', default=8, type=int, help='chunk size. for image, chunk x chunk will be the actual chunk size')\n",
    "    parser.add_argument('--cuda', default=True, type=str2bool, help = 'enable cuda')\n",
    "    parser.add_argument('--out_dir', type=str, default='./result/saliency/BUSI_8_50_2', help='Result directory path')\n",
    "    parser.add_argument('--K', type=int, default=50, help='dimension of encoding Z')\n",
    "    \n",
    "    args = parser.parse_args([])\n",
    "    \n",
    "    args.cuda = args.cuda and torch.cuda.is_available()\n",
    "    if args.cuda:\n",
    "        print(\"Using GPU for acceleration\")\n",
    "    else:\n",
    "        print(\"Using CPU for computation\")\n",
    "    \n",
    "    print('Input data: {}'.format(args.dataset))\n",
    "\n",
    "    return args\n",
    "#%%  \n",
    "def main():\n",
    "     \n",
    "    args = parse_args()\n",
    "\n",
    "    if not os.path.exists(args.out_dir):\n",
    "\n",
    "        os.makedirs(args.out_dir)\n",
    "\n",
    "    ## Data Loader\n",
    "    args.root = \"../mnist/dataset/Dataset_BUSI_AN/train/images\"\n",
    "    args.load_pred = False\n",
    "    args.device = torch.device(\"cuda\" if args.cuda else \"cpu\")\n",
    "    args.model_dir = '../' + args.dataset + '/models'\n",
    "    device = torch.device(\"cuda\" if args.cuda else \"cpu\")\n",
    "    \n",
    "    data_loader = return_data(args)\n",
    "    test_loader = data_loader['test']\n",
    "    \n",
    "    if 'mnist' in args.dataset:\n",
    "    \n",
    "        from Net import Net\n",
    "        ## load model\n",
    "        model = Net().to(device) \n",
    "        \n",
    "        args.word_idx = None\n",
    "        args.original_ncol = 224\n",
    "        args.original_nrow = 224\n",
    "        args.chunk_size = args.chunk_size if args.chunk_size > 0 else 1\n",
    "        assert np.remainder(args.original_nrow, args.chunk_size) == 0\n",
    "        args.filter_size = (args.chunk_size, args.chunk_size)\n",
    "        \n",
    "    else:\n",
    "    \n",
    "        raise UnknownDatasetError()\n",
    "            \n",
    "\n",
    "    model_name = Path(args.model_dir).joinpath(args.model_name)\n",
    "    model.load_state_dict(torch.load(model_name, map_location='cpu'))\n",
    "\n",
    "    if args.cuda:\n",
    "        model.cuda()\n",
    "\n",
    "    ## Prediction\n",
    "    test(args, model, device, test_loader, k=args.K)\n",
    "    \n",
    "    \n",
    "def test(args, model, device, test_loader, k, **kargs):\n",
    "    '''\n",
    "    k: the number of raw features selected\n",
    "    '''\n",
    "\n",
    "    model.eval()\n",
    "    # test_loss = 0\n",
    "    total_num = 0\n",
    "    total_num_ind = 0\n",
    "    # correct = 0\n",
    "    \n",
    "    correct_zeropadded = 0\n",
    "    precision_macro_zeropadded = 0  \n",
    "    precision_micro_zeropadded = 0\n",
    "    precision_weighted_zeropadded = 0\n",
    "    recall_macro_zeropadded = 0\n",
    "    recall_micro_zeropadded = 0\n",
    "    recall_weighted_zeropadded = 0\n",
    "    f1_macro_zeropadded = 0\n",
    "    f1_micro_zeropadded = 0\n",
    "    f1_weighted_zeropadded = 0\n",
    "\n",
    "    vmi_zeropadded_sum = 0\n",
    "    vmi_fidel_sum = 0\n",
    "    vmi_fidel_fixed_sum = 0\n",
    "    \n",
    "    correct_approx = 0\n",
    "    precision_macro_approx = 0\n",
    "    precision_micro_approx = 0\n",
    "    precision_weighted_approx = 0\n",
    "    recall_macro_approx = 0\n",
    "    recall_micro_approx = 0\n",
    "    recall_weighted_approx = 0\n",
    "    f1_macro_approx = 0\n",
    "    f1_micro_approx = 0\n",
    "    f1_weighted_approx = 0\n",
    "    \n",
    "    correct_approx_fixed = 0\n",
    "    precision_macro_approx_fixed = 0\n",
    "    precision_micro_approx_fixed = 0\n",
    "    precision_weighted_approx_fixed = 0\n",
    "    recall_macro_approx_fixed = 0\n",
    "    recall_micro_approx_fixed = 0\n",
    "    recall_weighted_approx_fixed = 0\n",
    "    f1_macro_approx_fixed = 0\n",
    "    f1_micro_approx_fixed = 0\n",
    "    f1_weighted_approx_fixed = 0    \n",
    "        \n",
    "    is_cuda = args.cuda       \n",
    "                        \n",
    "    # outmode = \"TEST\"\n",
    "    #\n",
    "    # if outfile:\n",
    "    #    assert kargs['outmode'] in ['train', 'test', 'valid']\n",
    "    #    outmode = kargs['outmode']\n",
    "    \n",
    "    # with torch.no_grad():\n",
    "        \n",
    "    # predictions = []\n",
    "    # predictions_idx = []\n",
    "\n",
    "    for idx, batch in enumerate(test_loader):  # (data, target, _, _)\n",
    "\n",
    "        if 'mnist' in args.dataset:\n",
    "            num_labels = 2\n",
    "            data = batch[0]\n",
    "            target = batch[1]\n",
    "            idx_list = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 17,]\n",
    "            \n",
    "        else:\n",
    "            raise UnknownDatasetError()\n",
    "                   \n",
    "        data, target = data.to(device), target.to(device)\n",
    "        output_all, output_all2 = model(data)\n",
    "        # test_loss += F.cross_entropy(output_all, target, reduction = 'sum').item()\n",
    "        pred = output_all.max(-1, keepdim=True)[1] # get the index of the max log-probability\n",
    "        # correct += pred.eq(target.view_as(pred)).sum().item()\n",
    "        total_num += 1\n",
    "        total_num_ind += data.size(0)\n",
    "           \n",
    "        for i in range(data.size(0)):\n",
    "            \n",
    "            if 'mnist' in args.dataset:\n",
    "            \n",
    "                ## Calculate Gradient\n",
    "                input = Variable(data[i:(i+1)], requires_grad=True)\n",
    "                output, _ = model(input)\n",
    "                output = torch.max(output)\n",
    "\n",
    "                if args.method == 'saliency':\n",
    "                    grad, = torch.autograd.grad(output, input, retain_graph=True)\n",
    "                    \n",
    "                elif args.method == 'smoothgrad':\n",
    "                    grad_ft = SmoothGrad(pretrained_model=model,\n",
    "                                         is_cuda=args.cuda,\n",
    "                                         n_samples=2,\n",
    "                                         magnitude=False)\n",
    "                    grad = grad_ft(input, index=None)\n",
    "                    \n",
    "                else:\n",
    "                    UnknownModelError()\n",
    "                    \n",
    "                ## Select Variables\n",
    "                grad_size = grad.size()\n",
    "                if args.chunk_size > 1:\n",
    "                    grad_chunk = F.avg_pool2d(torch.abs(grad), kernel_size=args.filter_size, stride=args.filter_size, padding=0)\n",
    "                    _, index_chunk = torch.abs(grad_chunk.view(grad_size[0], grad_size[1], -1)).topk(k, dim=-1) # index_chunk:[1, 1, 30]\n",
    "\n",
    "#                    if args.method == 'saliency':    \n",
    "#                        _, index_chunk = torch.abs(grad_chunk.view(grad_size[0], grad_size[1], -1)).topk(k, dim=-1)\n",
    "#                    \n",
    "#                    elif args.method == 'taylor':\n",
    "#                        _, index_chunk = torch.abs(grad_chunk.view(grad_size[0], grad_size[1], -1) * input.view())\n",
    "                    #                        .topk(k, dim = -1)\n",
    "#                        approx = torch.addcmul(torch.zeros(1), value = 1, \n",
    "#                                               tensor1 = grad_chunk.view(grad_size[0], grad_size[1], -1), \n",
    "#                                               tensor2 = input.unsqueeze(-1).view(grad_size[0], grad_size[1], -1), out=None)\n",
    "#                \n",
    "#                    else:\n",
    "#                        UnknownModelError()\n",
    "                    \n",
    "                    index = index_transfer(dataset=args.dataset,\n",
    "                                           idx=index_chunk,\n",
    "                                           filter_size=args.filter_size,\n",
    "                                           original_nrow=args.original_nrow,\n",
    "                                           original_ncol=args.original_ncol,\n",
    "                                           is_cuda=args.cuda).output.unsqueeze(1)\n",
    "                else:\n",
    "                    grad_chunk = grad\n",
    "                    _, index = torch.abs(grad_chunk.view(grad_size[0], grad_size[1], grad_size[2] * grad_size[3])).topk(k, dim=-1)\n",
    "\n",
    "                ## Approximation\n",
    "                grad_selected = grad.view(grad_size[0], grad_size[1], grad_size[2] * grad_size[3])[:, :, index[0][0].type(torch.long)]\n",
    "                data_selected = input.view(grad_size[0], grad_size[1], grad_size[2] * grad_size[3])[:, :, index[0][0].type(torch.long)]\n",
    "            \n",
    "            else:\n",
    "            \n",
    "                raise UnknownDatasetError()    \n",
    "\n",
    "            if i == 0:\n",
    "                grad_all = grad\n",
    "                index_all = index\n",
    "                grad_selected_all = grad_selected\n",
    "                data_selected_all = data_selected\n",
    "            else:\n",
    "                grad_all = torch.cat((grad_all, grad), dim = 0) \n",
    "                index_all = torch.cat((index_all, index), dim = 0)\n",
    "                grad_selected_all = torch.cat((grad_selected_all, grad_selected), dim = 0)\n",
    "                data_selected_all = torch.cat((data_selected_all, data_selected), dim = 0)\n",
    "                \n",
    "        if 'mnist' in args.dataset:\n",
    "            data_size = data.size()\n",
    "            binary_selected_all = idxtobool(index_all, [data_size[0], data_size[1], data_size[2] * data_size[3]], is_cuda)            \n",
    "            data_zeropadded = torch.addcmul(torch.zeros(1), value=1, tensor1=binary_selected_all.view(data_size).type(torch.FloatTensor), tensor2=data.type(torch.FloatTensor), out=None)\n",
    "        \n",
    "        else:\n",
    "            raise UnknownDatasetError()\n",
    "\n",
    "        # Post-hoc Accuracy (zero-padded accuracy)\n",
    "        output_zeropadded, output_zeropadded2 = model(cuda(data_zeropadded, is_cuda))             \n",
    "        pred_zeropadded = output_zeropadded.max(1, keepdim=True)[1]  # get the index of the max log-probability\n",
    "        correct_zeropadded += pred_zeropadded.eq(pred).sum().item()\n",
    "       \n",
    "        pred, pred_zeropadded = pred.cpu(), pred_zeropadded.cpu()\n",
    "        precision_macro_zeropadded += precision_score(pred, pred_zeropadded, average='macro')\n",
    "        precision_micro_zeropadded += precision_score(pred, pred_zeropadded, average='micro')\n",
    "        precision_weighted_zeropadded += precision_score(pred, pred_zeropadded, average='weighted')\n",
    "        recall_macro_zeropadded += recall_score(pred, pred_zeropadded, average='macro')\n",
    "        recall_micro_zeropadded += recall_score(pred, pred_zeropadded, average='micro')\n",
    "        recall_weighted_zeropadded += recall_score(pred, pred_zeropadded, average='weighted')\n",
    "        f1_macro_zeropadded += f1_score(pred, pred_zeropadded, average='macro')\n",
    "        f1_micro_zeropadded += f1_score(pred, pred_zeropadded, average='micro')\n",
    "        f1_weighted_zeropadded += f1_score(pred, pred_zeropadded, average='weighted')\n",
    "\n",
    "        ## Variational Mutual Information            \n",
    "        vmi = torch.sum(torch.addcmul(torch.zeros(1), value=1,\n",
    "                                      tensor1=torch.exp(output_all).type(torch.FloatTensor),\n",
    "                                      tensor2 = output_zeropadded.type(torch.FloatTensor) - torch.logsumexp(output_all, dim = 0).unsqueeze(0).expand(output_zeropadded.size()).type(torch.FloatTensor) + torch.log(torch.tensor(output_all.size(0)).type(torch.FloatTensor)),\n",
    "                                      #tensor2 = output_zeropadded.type(torch.FloatTensor) - torch.sum(output_all, dim = -1).unsqueeze(-1).expand(output_zeropadded.size()).type(torch.FloatTensor),\n",
    "                                      out=None), dim = -1)\n",
    "        vmi_zeropadded_sum += vmi.sum().item()\n",
    "\n",
    "        ## Approximation Fidelity (prediction performance)\n",
    "        for outidx in range(num_labels):\n",
    "\n",
    "            for i in range(data.size(0)):\n",
    "\n",
    "                if 'mnist' in args.dataset:\n",
    "                \n",
    "                    ## Calculate Gradient\n",
    "                    input = Variable(data[i:(i+1), :, :, :], requires_grad = True) \n",
    "                    output, output2 = model(input)\n",
    "                    output = output[0][outidx]\n",
    "                    if args.method == 'saliency':\n",
    "                        grad, = torch.autograd.grad(output, input, retain_graph = True)\n",
    "#                         autograd.grad(outputs=b,inputs=a,grad_outputs=torch.ones_like(a))\n",
    "                        \n",
    "                    elif args.method == 'smoothgrad':\n",
    "                        grad_ft = SmoothGrad(pretrained_model = model, \n",
    "                                             is_cuda = args.cuda,\n",
    "                                             n_samples = 2,\n",
    "                                             magnitude = False)\n",
    "                        grad = grad_ft(input, index = None)\n",
    "                        \n",
    "                    else:\n",
    "                        UnknownModelError()\n",
    "\n",
    "                    ## Select Variables\n",
    "                    grad_size = grad.size()\n",
    "                    if args.chunk_size > 1:\n",
    "                        grad_chunk = F.avg_pool2d(torch.abs(grad), kernel_size = args.filter_size, stride = args.filter_size, padding = 0)\n",
    "                        _, index_chunk = torch.abs(grad_chunk.view(grad_size[0], grad_size[1], -1)).topk(k, dim = -1)\n",
    "                        index = index_transfer(dataset = args.dataset,\n",
    "                                                     idx = index_chunk, \n",
    "                                                     filter_size = args.filter_size,\n",
    "                                                     original_nrow = args.original_nrow,\n",
    "                                                     original_ncol = args.original_ncol, \n",
    "                                                     is_cuda = args.cuda).output.unsqueeze(1)\n",
    "                    else:\n",
    "                        grad_chunk = grad\n",
    "                        _, index = torch.abs(grad_chunk.view(grad_size[0], grad_size[1], grad_size[2] * grad_size[3])).topk(k, dim=-1)\n",
    "                    \n",
    "                    ## Approximation\n",
    "                    grad_selected = grad.view(grad_size[0], grad_size[1], grad_size[2] * grad_size[3])[:, :, index[0][0].type(torch.long)]\n",
    "                    data_selected = input.view(grad_size[0], grad_size[1], grad_size[2] * grad_size[3])[:, :, index[0][0].type(torch.long)]\n",
    "                    \n",
    "                else:\n",
    "                \n",
    "                    raise UnknownDatasetError()    \n",
    "                # print(i)\n",
    "                if i == 0:\n",
    "                    grad_all = grad\n",
    "                    index_all = index\n",
    "                    grad_selected_all = grad_selected\n",
    "                    data_selected_all = data_selected\n",
    "                else:\n",
    "                    grad_all = torch.cat((grad_all, grad), dim = 0) \n",
    "                    index_all = torch.cat((index_all, index), dim = 0)\n",
    "                    grad_selected_all = torch.cat((grad_selected_all, grad_selected), dim = 0)\n",
    "                    data_selected_all = torch.cat((data_selected_all, data_selected), dim = 0)\n",
    "            \n",
    "            if 'mnist' in args.dataset:\n",
    "            \n",
    "                approx = torch.addcmul(torch.zeros(1), value = 1, tensor1 = grad_all.view(data_size[0], data_size[1], data_size[2] * data_size[3]).type(torch.FloatTensor), tensor2 = data.view(data_size[0], data_size[1], data_size[2] * data_size[3]).type(torch.FloatTensor), out=None)\n",
    "                approx = torch.exp(torch.sum(approx, dim = -1))##squeeze(-1)\n",
    "                approx_fixed = torch.addcmul(torch.zeros(1), value=1, tensor1 = grad_selected_all.type(torch.FloatTensor) , tensor2 = data_selected_all.type(torch.FloatTensor), out=None)\n",
    "                approx_fixed = torch.exp(torch.sum(approx_fixed, dim = -1)) #.squeeze(-1)\n",
    "                \n",
    "            else:\n",
    "                \n",
    "                raise UnknownDatasetError()   \n",
    "            \n",
    "            if outidx == 0:\n",
    "                approx_all = approx\n",
    "                approx_fixed_all = approx_fixed\n",
    "            else:\n",
    "                approx_all = torch.cat((approx_all, approx), dim = 1)\n",
    "                approx_fixed_all = torch.cat((approx_fixed_all, approx_fixed), dim = 1)\n",
    "\n",
    "        pred = pred.type(torch.LongTensor)\n",
    "        pred_approx = approx_all.topk(1, dim = -1)[1]\n",
    "        pred_approx = pred_approx.type(torch.LongTensor)\n",
    "        pred_approx_fixed = approx_fixed_all.topk(1, dim = -1)[1]\n",
    "        pred_approx_fixed = pred_approx_fixed.type(torch.LongTensor)\n",
    "        pred_approx_logit = F.softmax(torch.log(approx_all), dim=1)\n",
    "        pred_approx_fixed_logit = F.softmax(torch.log(approx_fixed_all), dim = -1)\n",
    "  \n",
    "        correct_approx += pred_approx.eq(pred).sum().item()\n",
    "        precision_macro_approx += precision_score(pred, pred_approx, average = 'macro')  \n",
    "        precision_micro_approx += precision_score(pred, pred_approx, average = 'micro')  \n",
    "        precision_weighted_approx += precision_score(pred, pred_approx, average = 'weighted')\n",
    "        recall_macro_approx += recall_score(pred, pred_approx, average = 'macro')\n",
    "        recall_micro_approx += recall_score(pred, pred_approx, average = 'micro')\n",
    "        recall_weighted_approx += recall_score(pred, pred_approx, average = 'weighted')\n",
    "        f1_macro_approx += f1_score(pred, pred_approx, average = 'macro')\n",
    "        f1_micro_approx += f1_score(pred, pred_approx, average = 'micro')\n",
    "        f1_weighted_approx += f1_score(pred, pred_approx, average = 'weighted')\n",
    "        \n",
    "        correct_approx_fixed += pred_approx_fixed.eq(pred).sum().item()\n",
    "        precision_macro_approx_fixed += precision_score(pred, pred_approx_fixed, average = 'macro')  \n",
    "        precision_micro_approx_fixed += precision_score(pred, pred_approx_fixed, average = 'micro')  \n",
    "        precision_weighted_approx_fixed += precision_score(pred, pred_approx_fixed, average = 'weighted')\n",
    "        recall_macro_approx_fixed += recall_score(pred, pred_approx_fixed, average = 'macro')\n",
    "        recall_micro_approx_fixed += recall_score(pred, pred_approx_fixed, average = 'micro')\n",
    "        recall_weighted_approx_fixed += recall_score(pred, pred_approx_fixed, average = 'weighted')\n",
    "        f1_macro_approx_fixed += f1_score(pred, pred_approx_fixed, average = 'macro')\n",
    "        f1_micro_approx_fixed += f1_score(pred, pred_approx_fixed, average = 'micro')\n",
    "        f1_weighted_approx_fixed += f1_score(pred, pred_approx_fixed, average = 'weighted')    \n",
    "        \n",
    "        ## Variational Mutual Information    \n",
    "        vmi = torch.sum(torch.addcmul(torch.zeros(1), value=1,\n",
    "                                      tensor1=torch.exp(output_all).type(torch.FloatTensor),\n",
    "                                      tensor2 = pred_approx_logit.type(torch.FloatTensor) - torch.logsumexp(output_all, dim = 0).unsqueeze(0).expand(pred_approx_logit.size()).type(torch.FloatTensor) + torch.log(torch.tensor(output_all.size(0)).type(torch.FloatTensor)),\n",
    "                                      out=None), dim = -1)\n",
    "        vmi_fidel_sum += vmi.sum().item()\n",
    "\n",
    "        vmi = torch.sum(torch.addcmul(torch.zeros(1), value = 1, \n",
    "                                      tensor1 = torch.exp(output_all).type(torch.FloatTensor),\n",
    "                                      tensor2 = pred_approx_fixed_logit.type(torch.FloatTensor) - torch.logsumexp(output_all, dim = 0).unsqueeze(0).expand(pred_approx_fixed_logit.size()).type(torch.FloatTensor) + torch.log(torch.tensor(output_all.size(0)).type(torch.FloatTensor)),\n",
    "                                      out=None), dim = -1)\n",
    "        vmi_fidel_fixed_sum += vmi.sum().item()            \n",
    "\n",
    "#        #if (idx == 0 or idx == 200): ## figure\n",
    "#        if idx in idx_list:\n",
    "#\n",
    "#            filename = 'figure_saliency_' + args.dataset + '_' + str(k) + '_idx' + str(idx) + '.png'\n",
    "#            filename = Path(args.out_dir).joinpath(filename)\n",
    "#    \n",
    "#            #img = copy.deepcopy(grad_all)    \n",
    "#            img = copy.deepcopy(data)\n",
    "#            n_img = img.size(0)\n",
    "#            n_col = 8\n",
    "#            n_row = n_img // n_col + 1\n",
    "#    \n",
    "#            fig = plt.figure(figsize=(n_col * 1.5, n_row * 1.5)) \n",
    "#    \n",
    "#            for i in range(n_img):\n",
    "#    \n",
    "#                plt.subplot(n_row, n_col, 1 + i)\n",
    "#                plt.axis('off')\n",
    "#                # original image\n",
    "#                img0 = img[i].squeeze(0)#.numpy()\n",
    "#                plt.imshow(img0, cmap = 'autumn_r')\n",
    "#                # chunk selected\n",
    "#                img2 = img[i].view(-1)#.numpy()\n",
    "#                img2[index_all[i]] = cuda(torch.tensor(float('nan')), is_cuda)\n",
    "#                img2 = img2.view(img0.size())#.numpy()\n",
    "#                plt.title('BB {}, Apx {}'.format(pred[i].item(), pred[i].item()))\n",
    "#                plt.imshow(img2, cmap = 'gray')\n",
    "#    \n",
    "#            fig.subplots_adjust(wspace = 0.05, hspace = 0.35)       \n",
    "#            fig.savefig(filename)\n",
    "#            \n",
    "#            ## Save predictions\n",
    "#            #predictions.extend(pred.data.squeeze(-1).cpu().tolist())\n",
    "#            #predictions_idx.extend(idx.cpu().tolist())\n",
    "\n",
    "        #print(\"SAVED!!!!\")\n",
    "#         if idx in idx_list:\n",
    "        if idx in range(len(test_loader)):\n",
    "            # filename\n",
    "            filename = 'figure_'+ args.method + '_' + args.dataset + '_chunk' + str(args.chunk_size) + '_' + str(k) + '_idx' + str(idx) + '.png'\n",
    "            filename = Path(args.out_dir).joinpath(filename)\n",
    "            index_chunk = index_all\n",
    "            \n",
    "#            if args.chunk_size is not 1:\n",
    "#                \n",
    "#                index_chunk = index_transfer(dataset = args.dataset,\n",
    "#                                             idx = index_chunk, \n",
    "#                                             filter_size = args.filter_size,\n",
    "#                                             original_nrow = args.original_nrow,\n",
    "#                                             original_ncol = args.original_ncol, \n",
    "#                                             is_cuda = args.cuda).output\n",
    "            \n",
    "            save_batch(dataset = args.dataset, \n",
    "                       batch = data, label = target, label_pred = pred.squeeze(-1), label_approx = pred_approx_fixed.squeeze(-1),\n",
    "                       index = index_chunk, \n",
    "                       filename = filename, \n",
    "                       is_cuda = args.cuda,\n",
    "                       word_idx = args.word_idx).output\n",
    "#%%                                 \n",
    "    ## Post-hoc Accuracy (zero-padded accuracy)\n",
    "    accuracy_zeropadded = correct_zeropadded/total_num_ind\n",
    "    precision_macro_zeropadded = precision_macro_zeropadded/total_num\n",
    "    precision_micro_zeropadded = precision_micro_zeropadded/total_num\n",
    "    precision_weighted_zeropadded = precision_weighted_zeropadded/total_num\n",
    "    recall_macro_zeropadded = recall_macro_zeropadded/total_num\n",
    "    recall_micro_zeropadded = recall_micro_zeropadded/total_num\n",
    "    recall_weighted_zeropadded = recall_weighted_zeropadded/total_num\n",
    "    f1_macro_zeropadded = f1_macro_zeropadded/total_num\n",
    "    f1_micro_zeropadded = f1_micro_zeropadded/total_num\n",
    "    f1_weighted_zeropadded = f1_weighted_zeropadded/total_num\n",
    "    \n",
    "    ## VMI\n",
    "    vmi_zeropadded = vmi_zeropadded_sum/total_num_ind\n",
    "    vmi_fidel = vmi_fidel_sum / total_num_ind\n",
    "    vmi_fidel_fixed = vmi_fidel_fixed_sum / total_num_ind\n",
    "    \n",
    "    ## Approximation Fidelity (prediction performance)\n",
    "    accuracy_approx = correct_approx/total_num_ind\n",
    "    precision_macro_approx = precision_macro_approx/total_num\n",
    "    precision_micro_approx = precision_micro_approx/total_num\n",
    "    precision_weighted_approx = precision_weighted_approx/total_num\n",
    "    recall_macro_approx = recall_macro_approx/total_num\n",
    "    recall_micro_approx = recall_micro_approx/total_num\n",
    "    recall_weighted_approx = recall_weighted_approx/total_num\n",
    "    f1_macro_approx = f1_macro_approx/total_num\n",
    "    f1_micro_approx = f1_micro_approx/total_num\n",
    "    f1_weighted_approx = f1_weighted_approx/total_num\n",
    "    \n",
    "    accuracy_approx_fixed = correct_approx_fixed/total_num_ind\n",
    "    precision_macro_approx_fixed = precision_macro_approx_fixed/total_num\n",
    "    precision_micro_approx_fixed = precision_micro_approx_fixed/total_num\n",
    "    precision_weighted_approx_fixed = precision_weighted_approx_fixed/total_num\n",
    "    recall_macro_approx_fixed = recall_macro_approx_fixed/total_num\n",
    "    recall_micro_approx_fixed = recall_micro_approx_fixed/total_num\n",
    "    recall_weighted_approx_fixed = recall_weighted_approx_fixed/total_num\n",
    "    f1_macro_approx_fixed = f1_macro_approx_fixed/total_num\n",
    "    f1_micro_approx_fixed = f1_micro_approx_fixed/total_num\n",
    "    f1_weighted_approx_fixed = f1_weighted_approx/total_num\n",
    "\n",
    "    print('\\n\\n[VAL RESULT]\\n')\n",
    "    # Post-hoc Accuracy (zero-padded accuracy); \n",
    "\n",
    "    print('acc_zeropadded:{:.4f} avg_acc:{:.4f} avg_acc_fixed:{:.4f}'\n",
    "            .format(accuracy_zeropadded, accuracy_approx, accuracy_approx_fixed), end = '\\n')\n",
    "    print('precision_macro_zeropadded:{:.4f} precision_macro_approx:{:.4f} precision_macro_approx_fixed:{:.4f}'\n",
    "            .format(precision_macro_zeropadded, precision_macro_approx, precision_macro_approx_fixed), end = '\\n')   \n",
    "    print('precision_micro_zeropadded:{:.4f} precision_micro_approx:{:.4f} precision_micro_approx_fixed:{:.4f}'\n",
    "            .format(precision_micro_zeropadded, precision_micro_approx, precision_micro_approx_fixed), end = '\\n')   \n",
    "    print('recall_macro_zeropadded:{:.4f} recall_macro_approx:{:.4f} recall_macro_approx_fixed:{:.4f}'\n",
    "            .format(recall_macro_zeropadded, recall_macro_approx, recall_macro_approx_fixed), end = '\\n')   \n",
    "    print('recall_micro_zeropadded:{:.4f} recall_micro_approx:{:.4f} recall_micro_approx_fixed:{:.4f}'\n",
    "            .format(recall_micro_zeropadded, recall_micro_approx, recall_micro_approx_fixed), end = '\\n') \n",
    "    print('f1_macro_zeropadded:{:.4f} f1_macro_approx:{:.4f} f1_macro_approx_fixed:{:.4f}'\n",
    "            .format(f1_macro_zeropadded, f1_macro_approx, f1_macro_approx_fixed), end = '\\n')   \n",
    "    print('f1_micro_zeropadded:{:.4f} f1_micro_approx:{:.4f} f1_micro_approx_fixed:{:.4f}'\n",
    "            .format(f1_micro_zeropadded, f1_micro_approx_fixed, f1_micro_approx_fixed), end = '\\n') \n",
    "    print('vmi:{:.4f} vmi_fixed:{:.4f} vmi_zeropadded:{:.4f}'.format(vmi_fidel, vmi_fidel_fixed, vmi_zeropadded), end = '\\n')\n",
    "    print()\n",
    "    \n",
    "#%%\n",
    "#        if outfile:\n",
    "#            \n",
    "#            predictions = np.array(predictions)\n",
    "#            predictions_idx = np.array(predictions_idx)\n",
    "#            inds = predictions_idx.argsort()\n",
    "#            sorted_predictions = predictions[inds]\n",
    "#\n",
    "#            output_name = model_name + '_pred_' + outmode + '.pt'\n",
    "#            torch.save(sorted_predictions, Path(outfile_path).joinpath(output_name))                \n",
    "\n",
    "if __name__ == '__main__':\n",
    "    main()\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2faa4c16",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0831fd13",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
